/*
*  TCC - Tiny C Compiler
* 
*  Copyright (c) 2001-2004 Fabrice Bellard
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
*/


static const char tcc_keywords[] = 
#define DEF(id, str) str "\0"
#include "tcctok.h"
#undef DEF
;

/* WARNING: the content of this string encodes token numbers */
static char tok_two_chars[] = "<=\236>=\235!=\225&&\240||\241++\244--\242==\224<<\1>>\2+=\253-=\255*=\252/=\257%=\245&=\246^=\336|=\374->\313..\250##\266";

/* true if isid(c) || isnum(c) */
static unsigned char isidnum_table[256-CH_EOF];


struct macro_level
{
	struct macro_level *prev;
	int *p;
};

static void next_nomacro(void);
static void next_nomacro_spc(void);
static void macro_subst(TokenString *tok_str, Sym **nested_list,
						const int *macro_str, struct macro_level **can_read_stream);


/* allocate a new token */
static TokenSym *tok_alloc_new(TokenSym **pts, const char *str, int len)
{
	TokenSym *ts, **ptable;
	int i;

	if (tok_ident >= SYM_FIRST_ANOM) 
		error("memory full");

	/* expand token table if needed */
	i = tok_ident - TOK_IDENT;
	if ((i % TOK_ALLOC_INCR) == 0) {
		ptable = tcc_realloc(table_ident, (i + TOK_ALLOC_INCR) * sizeof(TokenSym *));
		if (!ptable)
			error("memory full");
		table_ident = ptable;
	}

	ts = tcc_malloc(sizeof(TokenSym) + len);
	table_ident[i] = ts;
	ts->tok = tok_ident++;
	ts->sym_define = NULL;
	ts->sym_label = NULL;
	ts->sym_struct = NULL;
	ts->sym_identifier = NULL;
	ts->len = len;
	ts->hash_next = NULL;
	memcpy(ts->str, str, len);
	ts->str[len] = '\0';
	*pts = ts;
	return ts;
}

#define TOK_HASH_INIT 1
#define TOK_HASH_FUNC(h, c) ((h) * 263 + (c))

/* find a token and add it if not found */
static TokenSym *tok_alloc(const char *str, int len)
{
	TokenSym *ts, **pts;
	int i;
	unsigned int h;

	h = TOK_HASH_INIT;
	for(i=0;i<len;i++)
		h = TOK_HASH_FUNC(h, ((unsigned char *)str)[i]);
	h &= (TOK_HASH_SIZE - 1);

	pts = &hash_ident[h];
	for(;;) {
		ts = *pts;
		if (!ts)
			break;
		if (ts->len == len && !memcmp(ts->str, str, len))
			return ts;
		pts = &(ts->hash_next);
	}
	return tok_alloc_new(pts, str, len);
}

/* XXX: buffer overflow */
/* XXX: float tokens */
char *get_tok_str(int v, CValue *cv)
{
	static char buf[STRING_MAX_SIZE + 1];
	static CString cstr_buf;
	CString *cstr;
	unsigned char *q;
	char *p;
	int i, len;

	/* NOTE: to go faster, we give a fixed buffer for small strings */
	cstr_reset(&cstr_buf);
	cstr_buf.data = buf;
	cstr_buf.size_allocated = sizeof(buf);
	p = buf;

	switch(v) {
case TOK_CINT:
case TOK_CUINT:
	/* XXX: not quite exact, but only useful for testing */
	sprintf(p, "%u", cv->ui);
	break;
case TOK_CLLONG:
case TOK_CULLONG:
	/* XXX: not quite exact, but only useful for testing  */
	sprintf(p, "%Lu", cv->ull);
	break;
case TOK_LCHAR:
	cstr_ccat(&cstr_buf, 'L');
case TOK_CCHAR:
	cstr_ccat(&cstr_buf, '\'');
	add_char(&cstr_buf, cv->i);
	cstr_ccat(&cstr_buf, '\'');
	cstr_ccat(&cstr_buf, '\0');
	break;
case TOK_PPNUM:
	cstr = cv->cstr;
	len = cstr->size - 1;
	for(i=0;i<len;i++)
		add_char(&cstr_buf, ((unsigned char *)cstr->data)[i]);
	cstr_ccat(&cstr_buf, '\0');
	break;
case TOK_LSTR:
	cstr_ccat(&cstr_buf, 'L');
case TOK_STR:
	cstr = cv->cstr;
	cstr_ccat(&cstr_buf, '\"');
	if (v == TOK_STR) {
		len = cstr->size - 1;
		for(i=0;i<len;i++)
			add_char(&cstr_buf, ((unsigned char *)cstr->data)[i]);
	} else {
		len = (cstr->size / sizeof(nwchar_t)) - 1;
		for(i=0;i<len;i++)
			add_char(&cstr_buf, ((nwchar_t *)cstr->data)[i]);
	}
	cstr_ccat(&cstr_buf, '\"');
	cstr_ccat(&cstr_buf, '\0');
	break;
case TOK_LT:
	v = '<';
	goto addv;
case TOK_GT:
	v = '>';
	goto addv;
case TOK_DOTS:
	return strcpy(p, "...");
case TOK_A_SHL:
	return strcpy(p, "<<=");
case TOK_A_SAR:
	return strcpy(p, ">>=");
default:
	if (v < TOK_IDENT) {
		/* search in two bytes table */
		q = tok_two_chars;
		while (*q) {
			if (q[2] == v) {
				*p++ = q[0];
				*p++ = q[1];
				*p = '\0';
				return buf;
			}
			q += 3;
		}
addv:
		*p++ = v;
		*p = '\0';
	} else if (v < tok_ident) {
		return table_ident[v - TOK_IDENT]->str;
	} else if (v >= SYM_FIRST_ANOM) {
		/* special name for anonymous symbol */
		sprintf(p, "L.%u", v - SYM_FIRST_ANOM);
	} else {
		/* should never happen */
		return NULL;
	}
	break;
	}
	return cstr_buf.data;
}

/* fill input buffer and peek next char */
static int tcc_peekc_slow(BufferedFile *bf)
{
	int len;
	/* only tries to read if really end of buffer */
	if (bf->buf_ptr >= bf->buf_end) {
		if (bf->fd != -1) {
#if defined(PARSE_DEBUG)
			len = 8;
#else
			len = IO_BUF_SIZE;
#endif
			len = _read(bf->fd, bf->buffer, len);
			if (len < 0)
				len = 0;
		} else {
			len = 0;
		}
		total_bytes += len;
		bf->buf_ptr = bf->buffer;
		bf->buf_end = bf->buffer + len;
		*bf->buf_end = CH_EOB;
	}
	if (bf->buf_ptr < bf->buf_end) {
		return bf->buf_ptr[0];
	} else {
		bf->buf_ptr = bf->buf_end;
		return CH_EOF;
	}
}

/* return the current character, handling end of block if necessary
(but not stray) */
static int handle_eob(void)
{
	return tcc_peekc_slow(file);
}

/* read next char from current input file and handle end of input buffer */
static inline void inp(void)
{
	ch = *(++(file->buf_ptr));
	/* end of buffer/file handling */
	if (ch == CH_EOB)
		ch = handle_eob();
}

/* handle '\[\r]\n' */
static int handle_stray_noerror(void)
{
	while (ch == '\\') {
		inp();
		if (ch == '\n') {
			file->line_num++;
			inp();
		} else if (ch == '\r') {
			inp();
			if (ch != '\n')
				goto fail;
			file->line_num++;
			inp();
		} else {
fail:
			return 1;
		}
	}
	return 0;
}

static void handle_stray(void)
{
	if (handle_stray_noerror())
		error("stray '\\' in program");
}

/* skip the stray and handle the \\n case. Output an error if
incorrect char after the stray */
static int handle_stray1(uint8_t *p)
{
	int c;

	if (p >= file->buf_end) {
		file->buf_ptr = p;
		c = handle_eob();
		p = file->buf_ptr;
		if (c == '\\')
			goto parse_stray;
	} else {
parse_stray:
		file->buf_ptr = p;
		ch = *p;
		handle_stray();
		p = file->buf_ptr;
		c = *p;
	}
	return c;
}

/* handle just the EOB case, but not stray */
#define PEEKC_EOB(c, p)\
{\
	p++;\
	c = *p;\
	if (c == '\\') {\
	file->buf_ptr = p;\
	c = handle_eob();\
	p = file->buf_ptr;\
	}\
}

/* handle the complicated stray case */
#define PEEKC(c, p)\
{\
	p++;\
	c = *p;\
	if (c == '\\') {\
	c = handle_stray1(p);\
	p = file->buf_ptr;\
	}\
}

/* input with '\[\r]\n' handling. Note that this function cannot
handle other characters after '\', so you cannot call it inside
strings or comments */
static void minp(void)
{
	inp();
	if (ch == '\\') 
		handle_stray();
}


/* single line C++ comments */
static uint8_t *parse_line_comment(uint8_t *p)
{
	int c;

	p++;
	for(;;) {
		c = *p;
redo:
		if (c == '\n' || c == CH_EOF) {
			break;
		} else if (c == '\\') {
			file->buf_ptr = p;
			c = handle_eob();
			p = file->buf_ptr;
			if (c == '\\') {
				PEEKC_EOB(c, p);
				if (c == '\n') {
					file->line_num++;
					PEEKC_EOB(c, p);
				} else if (c == '\r') {
					PEEKC_EOB(c, p);
					if (c == '\n') {
						file->line_num++;
						PEEKC_EOB(c, p);
					}
				}
			} else {
				goto redo;
			}
		} else {
			p++;
		}
	}
	return p;
}

/* C comments */
static uint8_t *parse_comment(uint8_t *p)
{
	int c;

	p++;
	for(;;) {
		/* fast skip loop */
		for(;;) {
			c = *p;
			if (c == '\n' || c == '*' || c == '\\')
				break;
			p++;
			c = *p;
			if (c == '\n' || c == '*' || c == '\\')
				break;
			p++;
		}
		/* now we can handle all the cases */
		if (c == '\n') {
			file->line_num++;
			p++;
		} else if (c == '*') {
			p++;
			for(;;) {
				c = *p;
				if (c == '*') {
					p++;
				} else if (c == '/') {
					goto end_of_comment;
				} else if (c == '\\') {
					file->buf_ptr = p;
					c = handle_eob();
					p = file->buf_ptr;
					if (c == '\\') {
						/* skip '\[\r]\n', otherwise just skip the stray */
						while (c == '\\') {
							PEEKC_EOB(c, p);
							if (c == '\n') {
								file->line_num++;
								PEEKC_EOB(c, p);
							} else if (c == '\r') {
								PEEKC_EOB(c, p);
								if (c == '\n') {
									file->line_num++;
									PEEKC_EOB(c, p);
								}
							} else {
								goto after_star;
							}
						}
					}
				} else {
					break;
				}
			}
after_star: ;
		} else {
			/* stray, eob or eof */
			file->buf_ptr = p;
			c = handle_eob();
			p = file->buf_ptr;
			if (c == CH_EOF) {
				error("unexpected end of file in comment");
			} else if (c == '\\') {
				p++;
			}
		}
	}
end_of_comment:
	p++;
	return p;
}

#define cinp minp

static inline void skip_spaces(void)
{
	while (is_space(ch))
		cinp();
}

static inline int check_space(int t, int *spc) 
{
	if (is_space(t)) {
		if (*spc) 
			return 1;
		*spc = 1;
	} else 
		*spc = 0;
	return 0;
}

/* parse a string without interpreting escapes */
static uint8_t *parse_pp_string(uint8_t *p,
								int sep, CString *str)
{
	int c;
	p++;
	for(;;) {
		c = *p;
		if (c == sep) {
			break;
		} else if (c == '\\') {
			file->buf_ptr = p;
			c = handle_eob();
			p = file->buf_ptr;
			if (c == CH_EOF) {
unterminated_string:
				/* XXX: indicate line number of start of string */
				error("missing terminating %c character", sep);
			} else if (c == '\\') {
				/* escape : just skip \[\r]\n */
				PEEKC_EOB(c, p);
				if (c == '\n') {
					file->line_num++;
					p++;
				} else if (c == '\r') {
					PEEKC_EOB(c, p);
					if (c != '\n')
						expect("'\n' after '\r'");
					file->line_num++;
					p++;
				} else if (c == CH_EOF) {
					goto unterminated_string;
				} else {
					if (str) {
						cstr_ccat(str, '\\');
						cstr_ccat(str, c);
					}
					p++;
				}
			}
		} else if (c == '\n') {
			file->line_num++;
			goto add_char;
		} else if (c == '\r') {
			PEEKC_EOB(c, p);
			if (c != '\n') {
				if (str)
					cstr_ccat(str, '\r');
			} else {
				file->line_num++;
				goto add_char;
			}
		} else {
add_char:
			if (str)
				cstr_ccat(str, c);
			p++;
		}
	}
	p++;
	return p;
}

/* skip block of text until #else, #elif or #endif. skip also pairs of
#if/#endif */
void preprocess_skip(void)
{
	int a, start_of_line, c, in_warn_or_error;
	uint8_t *p;

	p = file->buf_ptr;
	a = 0;
redo_start:
	start_of_line = 1;
	in_warn_or_error = 0;
	for(;;) {
redo_no_start:
		c = *p;
		switch(c) {
case ' ':
case '\t':
case '\f':
case '\v':
case '\r':
	p++;
	goto redo_no_start;
case '\n':
	file->line_num++;
	p++;
	goto redo_start;
case '\\':
	file->buf_ptr = p;
	c = handle_eob();
	if (c == CH_EOF) {
		expect("#endif");
	} else if (c == '\\') {
		ch = file->buf_ptr[0];
		handle_stray_noerror();
	}
	p = file->buf_ptr;
	goto redo_no_start;
	/* skip strings */
case '\"':
case '\'':
	if (in_warn_or_error)
		goto _default;
	p = parse_pp_string(p, c, NULL);
	break;
	/* skip comments */
case '/':
	if (in_warn_or_error)
		goto _default;
	file->buf_ptr = p;
	ch = *p;
	minp();
	p = file->buf_ptr;
	if (ch == '*') {
		p = parse_comment(p);
	} else if (ch == '/') {
		p = parse_line_comment(p);
	}
	break;
case '#':
	p++;
	if (start_of_line) {
		file->buf_ptr = p;
		next_nomacro();
		p = file->buf_ptr;
		if (a == 0 && 
			(tok == TOK_ELSE || tok == TOK_ELIF || tok == TOK_ENDIF))
			goto the_end;
		if (tok == TOK_IF || tok == TOK_IFDEF || tok == TOK_IFNDEF)
			a++;
		else if (tok == TOK_ENDIF)
			a--;
		else if( tok == TOK_ERROR || tok == TOK_WARNING)
			in_warn_or_error = 1;
	}
	break;
_default:
default:
	p++;
	break;
		}
		start_of_line = 0;
	}
the_end: ;
	file->buf_ptr = p;
}

/* ParseState handling */

/* XXX: currently, no include file info is stored. Thus, we cannot display
accurate messages if the function or data definition spans multiple
files */

/* save current parse state in 's' */
void save_parse_state(ParseState *s)
{
	s->line_num = file->line_num;
	s->macro_ptr = macro_ptr;
	s->tok = tok;
	s->tokc = tokc;
}

/* restore parse state from 's' */
void restore_parse_state(ParseState *s)
{
	file->line_num = s->line_num;
	macro_ptr = s->macro_ptr;
	tok = s->tok;
	tokc = s->tokc;
}

/* return the number of additional 'ints' necessary to store the
token */
static inline int tok_ext_size(int t)
{
	switch(t) {
		/* 4 bytes */
case TOK_CINT:
case TOK_CUINT:
case TOK_CCHAR:
case TOK_LCHAR:
case TOK_CFLOAT:
case TOK_LINENUM:
	return 1;
case TOK_STR:
case TOK_LSTR:
case TOK_PPNUM:
	error("unsupported token");
	return 1;
case TOK_CDOUBLE:
case TOK_CLLONG:
case TOK_CULLONG:
	return 2;
case TOK_CLDOUBLE:
	return LDOUBLE_SIZE / 4;
default:
	return 0;
	}
}

/* token string handling */

static inline void tok_str_new(TokenString *s)
{
	s->str = NULL;
	s->len = 0;
	s->allocated_len = 0;
	s->last_line_num = -1;
}

static void tok_str_free(int *str)
{
	tcc_free(str);
}

static int *tok_str_realloc(TokenString *s)
{
	int *str, len;

	if (s->allocated_len == 0) {
		len = 8;
	} else {
		len = s->allocated_len * 2;
	}
	str = tcc_realloc(s->str, len * sizeof(int));
	if (!str)
		error("memory full");
	s->allocated_len = len;
	s->str = str;
	return str;
}

static void tok_str_add(TokenString *s, int t)
{
	int len, *str;

	len = s->len;
	str = s->str;
	if (len >= s->allocated_len)
		str = tok_str_realloc(s);
	str[len++] = t;
	s->len = len;
}

static void tok_str_add2(TokenString *s, int t, CValue *cv)
{
	int len, *str;

	len = s->len;
	str = s->str;

	/* allocate space for worst case */
	if (len + TOK_MAX_SIZE > s->allocated_len)
		str = tok_str_realloc(s);
	str[len++] = t;
	switch(t) {
case TOK_CINT:
case TOK_CUINT:
case TOK_CCHAR:
case TOK_LCHAR:
case TOK_CFLOAT:
case TOK_LINENUM:
	str[len++] = cv->tab[0];
	break;
case TOK_PPNUM:
case TOK_STR:
case TOK_LSTR:
	{
		int nb_words;
		CString *cstr;

		nb_words = (sizeof(CString) + cv->cstr->size + 3) >> 2;
		while ((len + nb_words) > s->allocated_len)
			str = tok_str_realloc(s);
		cstr = (CString *)(str + len);
		cstr->data = NULL;
		cstr->size = cv->cstr->size;
		cstr->data_allocated = NULL;
		cstr->size_allocated = cstr->size;
		memcpy((char *)cstr + sizeof(CString), 
			cv->cstr->data, cstr->size);
		len += nb_words;
	}
	break;
case TOK_CDOUBLE:
case TOK_CLLONG:
case TOK_CULLONG:
#if LDOUBLE_SIZE == 8
case TOK_CLDOUBLE:
#endif
	str[len++] = cv->tab[0];
	str[len++] = cv->tab[1];
	break;
#if LDOUBLE_SIZE == 12
case TOK_CLDOUBLE:
	str[len++] = cv->tab[0];
	str[len++] = cv->tab[1];
	str[len++] = cv->tab[2];
#elif LDOUBLE_SIZE == 16
case TOK_CLDOUBLE:
	str[len++] = cv->tab[0];
	str[len++] = cv->tab[1];
	str[len++] = cv->tab[2];
	str[len++] = cv->tab[3];
#elif LDOUBLE_SIZE != 8
#error add long double size support
#endif
	break;
default:
	break;
	}
	s->len = len;
}

/* add the current parse token in token string 's' */
static void tok_str_add_tok(TokenString *s)
{
	CValue cval;

	/* save line number info */
	if (file->line_num != s->last_line_num) {
		s->last_line_num = file->line_num;
		cval.i = s->last_line_num;
		tok_str_add2(s, TOK_LINENUM, &cval);
	}
	tok_str_add2(s, tok, &tokc);
}

#if LDOUBLE_SIZE == 16
#define LDOUBLE_GET(p, cv)                      \
	cv.tab[0] = p[0];                       \
	cv.tab[1] = p[1];                       \
	cv.tab[2] = p[2];                       \
	cv.tab[3] = p[3];
#elif LDOUBLE_SIZE == 12
#define LDOUBLE_GET(p, cv)                      \
	cv.tab[0] = p[0];                       \
	cv.tab[1] = p[1];                       \
	cv.tab[2] = p[2];
#elif LDOUBLE_SIZE == 8
#define LDOUBLE_GET(p, cv)                      \
	cv.tab[0] = p[0];                       \
	cv.tab[1] = p[1];
#else
#error add long double size support
#endif


/* get a token from an integer array and increment pointer
accordingly. we code it as a macro to avoid pointer aliasing. */
#define TOK_GET(t, p, cv)                       \
{                                               \
	t = *p++;                                   \
	switch(t) {                                 \
	case TOK_CINT:                              \
	case TOK_CUINT:                             \
	case TOK_CCHAR:                             \
	case TOK_LCHAR:                             \
	case TOK_CFLOAT:                            \
	case TOK_LINENUM:                           \
	cv.tab[0] = *p++;                       \
	break;                                  \
	case TOK_STR:                               \
	case TOK_LSTR:                              \
	case TOK_PPNUM:                             \
	cv.cstr = (CString *)p;                 \
	cv.cstr->data = (char *)p + sizeof(CString);\
	p += (sizeof(CString) + cv.cstr->size + 3) >> 2;\
	break;                                  \
	case TOK_CDOUBLE:                           \
	case TOK_CLLONG:                            \
	case TOK_CULLONG:                           \
	cv.tab[0] = p[0];                       \
	cv.tab[1] = p[1];                       \
	p += 2;                                 \
	break;                                  \
	case TOK_CLDOUBLE:                          \
	LDOUBLE_GET(p, cv);                     \
	p += LDOUBLE_SIZE / 4;                  \
	break;                                  \
	default:                                    \
	break;                                  \
}                                           \
}

/* defines handling */
static inline void define_push(int v, int macro_type, int *str, Sym *first_arg)
{
	Sym *s;

	s = sym_push2(&define_stack, v, macro_type, (long)str);
	s->next = first_arg;
	table_ident[v - TOK_IDENT]->sym_define = s;
}

/* undefined a define symbol. Its name is just set to zero */
static void define_undef(Sym *s)
{
	int v;
	v = s->v;
	if (v >= TOK_IDENT && v < tok_ident)
		table_ident[v - TOK_IDENT]->sym_define = NULL;
	s->v = 0;
}

static inline Sym *define_find(int v)
{
	v -= TOK_IDENT;
	if ((unsigned)v >= (unsigned)(tok_ident - TOK_IDENT))
		return NULL;
	return table_ident[v]->sym_define;
}

/* free define stack until top reaches 'b' */
static void free_defines(Sym *b)
{
	Sym *top, *top1;
	int v;

	top = define_stack;
	while (top != b) {
		top1 = top->prev;
		/* do not free args or predefined defines */
		if (top->c)
			tok_str_free((int *)top->c);
		v = top->v;
		if (v >= TOK_IDENT && v < tok_ident)
			table_ident[v - TOK_IDENT]->sym_define = NULL;
		sym_free(top);
		top = top1;
	}
	define_stack = b;
}

/* label lookup */
static Sym *label_find(int v)
{
	v -= TOK_IDENT;
	if ((unsigned)v >= (unsigned)(tok_ident - TOK_IDENT))
		return NULL;
	return table_ident[v]->sym_label;
}

static Sym *label_push(Sym **ptop, int v, int flags)
{
	Sym *s, **ps;
	s = sym_push2(ptop, v, 0, 0);
	s->r = flags;
	ps = &table_ident[v - TOK_IDENT]->sym_label;
	if (ptop == &global_label_stack) {
		/* modify the top most local identifier, so that
		sym_identifier will point to 's' when popped */
		while (*ps != NULL)
			ps = &(*ps)->prev_tok;
	}
	s->prev_tok = *ps;
	*ps = s;
	return s;
}

/* pop labels until element last is reached. Look if any labels are
undefined. Define symbols if '&&label' was used. */
static void label_pop(Sym **ptop, Sym *slast)
{
	Sym *s, *s1;
	for(s = *ptop; s != slast; s = s1) {
		s1 = s->prev;
		if (s->r == LABEL_DECLARED) {
			warning("label '%s' declared but not used", get_tok_str(s->v, NULL));
		} else if (s->r == LABEL_FORWARD) {
			error("label '%s' used but not defined",
				get_tok_str(s->v, NULL));
		} else {
			if (s->c) {
				/* define corresponding symbol. A size of
				1 is put. */
				put_extern_sym(s, cur_text_section, (long)s->next, 1);
			}
		}
		/* remove label */
		table_ident[s->v - TOK_IDENT]->sym_label = s->prev_tok;
		sym_free(s);
	}
	*ptop = slast;
}

/* eval an expression for #if/#elif */
static int expr_preprocess(void)
{
	int c, t;
	TokenString str;

	tok_str_new(&str);
	while (tok != TOK_LINEFEED && tok != TOK_EOF) {
		next(); /* do macro subst */
		if (tok == TOK_DEFINED) {
			next_nomacro();
			t = tok;
			if (t == '(') 
				next_nomacro();
			c = define_find(tok) != 0;
			if (t == '(')
				next_nomacro();
			tok = TOK_CINT;
			tokc.i = c;
		} else if (tok >= TOK_IDENT) {
			/* if undefined macro */
			tok = TOK_CINT;
			tokc.i = 0;
		}
		tok_str_add_tok(&str);
	}
	tok_str_add(&str, -1); /* simulate end of file */
	tok_str_add(&str, 0);
	/* now evaluate C constant expression */
	macro_ptr = str.str;
	next();
	c = expr_const();
	macro_ptr = NULL;
	tok_str_free(str.str);
	return c != 0;
}

#if defined(PARSE_DEBUG) || defined(PP_DEBUG)
static void tok_print(int *str)
{
	int t;
	CValue cval;

	printf("<");
	while (1) {
		TOK_GET(t, str, cval);
		if (!t)
			break;
		printf("%s", get_tok_str(t, &cval));
	}
	printf(">\n");
}
#endif

/* parse after #define */
static void parse_define(void)
{
	Sym *s, *first, **ps;
	int v, t, varg, is_vaargs, spc;
	TokenString str;

	v = tok;
	if (v < TOK_IDENT)
		error("invalid macro name '%s'", get_tok_str(tok, &tokc));
	/* XXX: should check if same macro (ANSI) */
	first = NULL;
	t = MACRO_OBJ;
	/* '(' must be just after macro definition for MACRO_FUNC */
	next_nomacro_spc();
	if (tok == '(') {
		next_nomacro();
		ps = &first;
		while (tok != ')') {
			varg = tok;
			next_nomacro();
			is_vaargs = 0;
			if (varg == TOK_DOTS) {
				varg = TOK___VA_ARGS__;
				is_vaargs = 1;
			} else if (tok == TOK_DOTS && gnu_ext) {
				is_vaargs = 1;
				next_nomacro();
			}
			if (varg < TOK_IDENT)
				error("badly punctuated parameter list");
			s = sym_push2(&define_stack, varg | SYM_FIELD, is_vaargs, 0);
			*ps = s;
			ps = &s->next;
			if (tok != ',')
				break;
			next_nomacro();
		}
		if (tok == ')')
			next_nomacro_spc();
		t = MACRO_FUNC;
	}
	tok_str_new(&str);
	spc = 2;
	/* EOF testing necessary for '-D' handling */
	while (tok != TOK_LINEFEED && tok != TOK_EOF) {
		/* remove spaces around ## and after '#' */        
		if (TOK_TWOSHARPS == tok) {
			if (1 == spc)
				--str.len;
			spc = 2;
		} else if ('#' == tok) {
			spc = 2;
		} else if (check_space(tok, &spc)) {
			goto skip;
		}
		tok_str_add2(&str, tok, &tokc);
skip:
		next_nomacro_spc();
	}
	if (spc == 1)
		--str.len; /* remove trailing space */
	tok_str_add(&str, 0);
#ifdef PP_DEBUG
	printf("define %s %d: ", get_tok_str(v, NULL), t);
	tok_print(str.str);
#endif
	define_push(v, t, str.str, first);
}

static inline int hash_cached_include(int type, const char *filename)
{
	const unsigned char *s;
	unsigned int h;

	h = TOK_HASH_INIT;
	h = TOK_HASH_FUNC(h, type);
	s = filename;
	while (*s) {
		h = TOK_HASH_FUNC(h, *s);
		s++;
	}
	h &= (CACHED_INCLUDES_HASH_SIZE - 1);
	return h;
}

/* XXX: use a token or a hash table to accelerate matching ? */
static CachedInclude *search_cached_include(TCCState *s1,
											int type, const char *filename)
{
	CachedInclude *e;
	int i, h;
	h = hash_cached_include(type, filename);
	i = s1->cached_includes_hash[h];
	for(;;) {
		if (i == 0)
			break;
		e = s1->cached_includes[i - 1];
		if (e->type == type && !PATHCMP(e->filename, filename))
			return e;
		i = e->hash_next;
	}
	return NULL;
}

static inline void add_cached_include(TCCState *s1, int type, 
									  const char *filename, int ifndef_macro)
{
	CachedInclude *e;
	int h;

	if (search_cached_include(s1, type, filename))
		return;
#ifdef INC_DEBUG
	printf("adding cached '%s' %s\n", filename, get_tok_str(ifndef_macro, NULL));
#endif
	e = tcc_malloc(sizeof(CachedInclude) + strlen(filename));
	if (!e)
		return;
	e->type = type;
	strcpy(e->filename, filename);
	e->ifndef_macro = ifndef_macro;
	dynarray_add((void ***)&s1->cached_includes, &s1->nb_cached_includes, e);
	/* add in hash table */
	h = hash_cached_include(type, filename);
	e->hash_next = s1->cached_includes_hash[h];
	s1->cached_includes_hash[h] = s1->nb_cached_includes;
}

static void pragma_parse(TCCState *s1)
{
	int val;

	next();
	if (tok == TOK_pack) {
		/*
		This may be:
		#pragma pack(1) // set
		#pragma pack() // reset to default
		#pragma pack(push,1) // push & set
		#pragma pack(pop) // restore previous
		*/
		next();
		skip('(');
		if (tok == TOK_ASM_pop) {
			next();
			if (s1->pack_stack_ptr <= s1->pack_stack) {
stk_error:
				error("out of pack stack");
			}
			s1->pack_stack_ptr--;
		} else {
			val = 0;
			if (tok != ')') {
				if (tok == TOK_ASM_push) {
					next();
					if (s1->pack_stack_ptr >= s1->pack_stack + PACK_STACK_SIZE - 1)
						goto stk_error;
					s1->pack_stack_ptr++;
					skip(',');
				}
				if (tok != TOK_CINT) {
pack_error:
					error("invalid pack pragma");
				}
				val = tokc.i;
				if (val < 1 || val > 16 || (val & (val - 1)) != 0)
					goto pack_error;
				next();
			}
			*s1->pack_stack_ptr = val;
			skip(')');
		}
	}
}

/* is_bof is true if first non space token at beginning of file */
static void preprocess(int is_bof)
{
	TCCState *s1 = tcc_state;
	int i, c, n, saved_parse_flags;
	char buf[1024], *q;
	Sym *s;

	saved_parse_flags = parse_flags;
	parse_flags = PARSE_FLAG_PREPROCESS | PARSE_FLAG_TOK_NUM | 
		PARSE_FLAG_LINEFEED;
	next_nomacro();
redo:
	switch(tok) {
case TOK_DEFINE:
	next_nomacro();
	parse_define();
	break;
case TOK_UNDEF:
	next_nomacro();
	s = define_find(tok);
	/* undefine symbol by putting an invalid name */
	if (s)
		define_undef(s);
	break;
case TOK_INCLUDE:
case TOK_INCLUDE_NEXT:
	ch = file->buf_ptr[0];
	/* XXX: incorrect if comments : use next_nomacro with a special mode */
	skip_spaces();
	if (ch == '<') {
		c = '>';
		goto read_name;
	} else if (ch == '\"') {
		c = ch;
read_name:
		inp();
		q = buf;
		while (ch != c && ch != '\n' && ch != CH_EOF) {
			if ((q - buf) < sizeof(buf) - 1)
				*q++ = ch;
			if (ch == '\\') {
				if (handle_stray_noerror() == 0)
					--q;
			} else
				inp();
		}
		*q = '\0';
		minp();
#if 0
		/* eat all spaces and comments after include */
		/* XXX: slightly incorrect */
		while (ch1 != '\n' && ch1 != CH_EOF)
			inp();
#endif
	} else {
		/* computed #include : either we have only strings or
		we have anything enclosed in '<>' */
		next();
		buf[0] = '\0';
		if (tok == TOK_STR) {
			while (tok != TOK_LINEFEED) {
				if (tok != TOK_STR) {
include_syntax:
					error("'#include' expects \"FILENAME\" or <FILENAME>");
				}
				pstrcat(buf, sizeof(buf), (char *)tokc.cstr->data);
				next();
			}
			c = '\"';
		} else {
			int len;
			while (tok != TOK_LINEFEED) {
				pstrcat(buf, sizeof(buf), get_tok_str(tok, &tokc));
				next();
			}
			len = strlen(buf);
			/* check syntax and remove '<>' */
			if (len < 2 || buf[0] != '<' || buf[len - 1] != '>')
				goto include_syntax;
			memmove(buf, buf + 1, len - 2);
			buf[len - 2] = '\0';
			c = '>';
		}
	}

	if (s1->include_stack_ptr >= s1->include_stack + INCLUDE_STACK_SIZE)
		error("#include recursion too deep");

	n = s1->nb_include_paths + s1->nb_sysinclude_paths;
	for (i = -2; i < n; ++i) {
		char buf1[sizeof file->filename];
		BufferedFile *f;
		CachedInclude *e;
		const char *path;
		int size;

		if (i == -2) {
			/* check absolute include path */
			if (!IS_ABSPATH(buf))
				continue;
			buf1[0] = 0;

		} else if (i == -1) {
			/* search in current dir if "header.h" */
			if (c != '\"')
				continue;
			size = tcc_basename(file->filename) - file->filename;
			memcpy(buf1, file->filename, size);
			buf1[size] = '\0';

		} else {
			/* search in all the include paths */
			if (i < s1->nb_include_paths)
				path = s1->include_paths[i];
			else
				path = s1->sysinclude_paths[i - s1->nb_include_paths];
			pstrcpy(buf1, sizeof(buf1), path);
			pstrcat(buf1, sizeof(buf1), "/");
		}

		pstrcat(buf1, sizeof(buf1), buf);

		e = search_cached_include(s1, c, buf1);
		if (e && define_find(e->ifndef_macro)) {
			/* no need to parse the include because the 'ifndef macro'
			is defined */
#ifdef INC_DEBUG
			printf("%s: skipping %s\n", file->filename, buf);
#endif
			f = NULL;
		}  else {
			f = tcc_open(s1, buf1);
			if (!f)
				continue;
		}

		if (tok == TOK_INCLUDE_NEXT) {
			tok = TOK_INCLUDE;
			if (f)
				tcc_close(f);
			continue;
		}

		if (!f)
			goto include_done;

#ifdef INC_DEBUG
		printf("%s: including %s\n", file->filename, buf1);
#endif

		/* XXX: fix current line init */
		/* push current file in stack */
		*s1->include_stack_ptr++ = file;
		f->inc_type = c;
		pstrcpy(f->inc_filename, sizeof(f->inc_filename), buf1);
		file = f;
		/* add include file debug info */
		if (tcc_state->do_debug) {
			put_stabs(file->filename, N_BINCL, 0, 0, 0);
		}
		tok_flags |= TOK_FLAG_BOF | TOK_FLAG_BOL;
		ch = file->buf_ptr[0];
		goto the_end;
	}
	error("include file '%s' not found", buf);
include_done:
	break;
case TOK_IFNDEF:
	c = 1;
	goto do_ifdef;
case TOK_IF:
	c = expr_preprocess();
	goto do_if;
case TOK_IFDEF:
	c = 0;
do_ifdef:
	next_nomacro();
	if (tok < TOK_IDENT)
		error("invalid argument for '#if%sdef'", c ? "n" : "");
	if (is_bof) {
		if (c) {
#ifdef INC_DEBUG
			printf("#ifndef %s\n", get_tok_str(tok, NULL));
#endif
			file->ifndef_macro = tok;
		}
	}
	c = (define_find(tok) != 0) ^ c;
do_if:
	if (s1->ifdef_stack_ptr >= s1->ifdef_stack + IFDEF_STACK_SIZE)
		error("memory full");
	*s1->ifdef_stack_ptr++ = c;
	goto test_skip;
case TOK_ELSE:
	if (s1->ifdef_stack_ptr == s1->ifdef_stack)
		error("#else without matching #if");
	if (s1->ifdef_stack_ptr[-1] & 2)
		error("#else after #else");
	c = (s1->ifdef_stack_ptr[-1] ^= 3);
	goto test_skip;
case TOK_ELIF:
	if (s1->ifdef_stack_ptr == s1->ifdef_stack)
		error("#elif without matching #if");
	c = s1->ifdef_stack_ptr[-1];
	if (c > 1)
		error("#elif after #else");
	/* last #if/#elif expression was true: we skip */
	if (c == 1)
		goto skip;
	c = expr_preprocess();
	s1->ifdef_stack_ptr[-1] = c;
test_skip:
	if (!(c & 1)) {
skip:
		preprocess_skip();
		is_bof = 0;
		goto redo;
	}
	break;
case TOK_ENDIF:
	if (s1->ifdef_stack_ptr <= file->ifdef_stack_ptr)
		error("#endif without matching #if");
	s1->ifdef_stack_ptr--;
	/* '#ifndef macro' was at the start of file. Now we check if
	an '#endif' is exactly at the end of file */
	if (file->ifndef_macro &&
		s1->ifdef_stack_ptr == file->ifdef_stack_ptr) {
			file->ifndef_macro_saved = file->ifndef_macro;
			/* need to set to zero to avoid false matches if another
			#ifndef at middle of file */
			file->ifndef_macro = 0;
			while (tok != TOK_LINEFEED)
				next_nomacro();
			tok_flags |= TOK_FLAG_ENDIF;
			goto the_end;
	}
	break;
case TOK_LINE:
	next();
	if (tok != TOK_CINT)
		error("#line");
	file->line_num = tokc.i - 1; /* the line number will be incremented after */
	next();
	if (tok != TOK_LINEFEED) {
		if (tok != TOK_STR)
			error("#line");
		pstrcpy(file->filename, sizeof(file->filename), 
			(char *)tokc.cstr->data);
	}
	break;
case TOK_ERROR:
case TOK_WARNING:
	c = tok;
	ch = file->buf_ptr[0];
	skip_spaces();
	q = buf;
	while (ch != '\n' && ch != CH_EOF) {
		if ((q - buf) < sizeof(buf) - 1)
			*q++ = ch;
		if (ch == '\\') {
			if (handle_stray_noerror() == 0)
				--q;
		} else
			inp();
	}
	*q = '\0';
	if (c == TOK_ERROR)
		error("#error %s", buf);
	else
		warning("#warning %s", buf);
	break;
case TOK_PRAGMA:
	pragma_parse(s1);
	break;
default:
	if (tok == TOK_LINEFEED || tok == '!' || tok == TOK_CINT) {
		/* '!' is ignored to allow C scripts. numbers are ignored
		to emulate cpp behaviour */
	} else {
		if (!(saved_parse_flags & PARSE_FLAG_ASM_COMMENTS))
			warning("Ignoring unknown preprocessing directive #%s", get_tok_str(tok, &tokc));
	}
	break;
	}
	/* ignore other preprocess commands or #! for C scripts */
	while (tok != TOK_LINEFEED)
		next_nomacro();
the_end:
	parse_flags = saved_parse_flags;
}

/* evaluate escape codes in a string. */
static void parse_escape_string(CString *outstr, const uint8_t *buf, int is_long)
{
	int c, n;
	const uint8_t *p;

	p = buf;
	for(;;) {
		c = *p;
		if (c == '\0')
			break;
		if (c == '\\') {
			p++;
			/* escape */
			c = *p;
			switch(c) {
case '0': case '1': case '2': case '3':
case '4': case '5': case '6': case '7':
	/* at most three octal digits */
	n = c - '0';
	p++;
	c = *p;
	if (isoct(c)) {
		n = n * 8 + c - '0';
		p++;
		c = *p;
		if (isoct(c)) {
			n = n * 8 + c - '0';
			p++;
		}
	}
	c = n;
	goto add_char_nonext;
case 'x':
case 'u':
case 'U':
	p++;
	n = 0;
	for(;;) {
		c = *p;
		if (c >= 'a' && c <= 'f')
			c = c - 'a' + 10;
		else if (c >= 'A' && c <= 'F')
			c = c - 'A' + 10;
		else if (isnum(c))
			c = c - '0';
		else
			break;
		n = n * 16 + c;
		p++;
	}
	c = n;
	goto add_char_nonext;
case 'a':
	c = '\a';
	break;
case 'b':
	c = '\b';
	break;
case 'f':
	c = '\f';
	break;
case 'n':
	c = '\n';
	break;
case 'r':
	c = '\r';
	break;
case 't':
	c = '\t';
	break;
case 'v':
	c = '\v';
	break;
case 'e':
	if (!gnu_ext)
		goto invalid_escape;
	c = 27;
	break;
case '\'':
case '\"':
case '\\': 
case '?':
	break;
default:
invalid_escape:
	if (c >= '!' && c <= '~')
		warning("unknown escape sequence: \'\\%c\'", c);
	else
		warning("unknown escape sequence: \'\\x%x\'", c);
	break;
			}
		}
		p++;
add_char_nonext:
		if (!is_long)
			cstr_ccat(outstr, c);
		else
			cstr_wccat(outstr, c);
	}
	/* add a trailing '\0' */
	if (!is_long)
		cstr_ccat(outstr, '\0');
	else
		cstr_wccat(outstr, '\0');
}

/* we use 64 bit numbers */
#define BN_SIZE 2

/* bn = (bn << shift) | or_val */
void bn_lshift(unsigned int *bn, int shift, int or_val)
{
	int i;
	unsigned int v;
	for(i=0;i<BN_SIZE;i++) {
		v = bn[i];
		bn[i] = (v << shift) | or_val;
		or_val = v >> (32 - shift);
	}
}

void bn_zero(unsigned int *bn)
{
	int i;
	for(i=0;i<BN_SIZE;i++) {
		bn[i] = 0;
	}
}

/* parse number in null terminated string 'p' and return it in the
current token */
void parse_number(const char *p)
{
	int b, t, shift, frac_bits, s, exp_val, ch;
	char *q;
	unsigned int bn[BN_SIZE];
	double d;

	/* number */
	q = token_buf;
	ch = *p++;
	t = ch;
	ch = *p++;
	*q++ = t;
	b = 10;
	if (t == '.') {
		goto float_frac_parse;
	} else if (t == '0') {
		if (ch == 'x' || ch == 'X') {
			q--;
			ch = *p++;
			b = 16;
		} else if (tcc_ext && (ch == 'b' || ch == 'B')) {
			q--;
			ch = *p++;
			b = 2;
		}
	}
	/* parse all digits. cannot check octal numbers at this stage
	because of floating point constants */
	while (1) {
		if (ch >= 'a' && ch <= 'f')
			t = ch - 'a' + 10;
		else if (ch >= 'A' && ch <= 'F')
			t = ch - 'A' + 10;
		else if (isnum(ch))
			t = ch - '0';
		else
			break;
		if (t >= b)
			break;
		if (q >= token_buf + STRING_MAX_SIZE) {
num_too_long:
			error("number too long");
		}
		*q++ = ch;
		ch = *p++;
	}
	if (ch == '.' ||
		((ch == 'e' || ch == 'E') && b == 10) ||
		((ch == 'p' || ch == 'P') && (b == 16 || b == 2))) {
			if (b != 10) {
				/* NOTE: strtox should support that for hexa numbers, but
				non ISOC99 libcs do not support it, so we prefer to do
				it by hand */
				/* hexadecimal or binary floats */
				/* XXX: handle overflows */
				*q = '\0';
				if (b == 16)
					shift = 4;
				else 
					shift = 2;
				bn_zero(bn);
				q = token_buf;
				while (1) {
					t = *q++;
					if (t == '\0') {
						break;
					} else if (t >= 'a') {
						t = t - 'a' + 10;
					} else if (t >= 'A') {
						t = t - 'A' + 10;
					} else {
						t = t - '0';
					}
					bn_lshift(bn, shift, t);
				}
				frac_bits = 0;
				if (ch == '.') {
					ch = *p++;
					while (1) {
						t = ch;
						if (t >= 'a' && t <= 'f') {
							t = t - 'a' + 10;
						} else if (t >= 'A' && t <= 'F') {
							t = t - 'A' + 10;
						} else if (t >= '0' && t <= '9') {
							t = t - '0';
						} else {
							break;
						}
						if (t >= b)
							error("invalid digit");
						bn_lshift(bn, shift, t);
						frac_bits += shift;
						ch = *p++;
					}
				}
				if (ch != 'p' && ch != 'P')
					expect("exponent");
				ch = *p++;
				s = 1;
				exp_val = 0;
				if (ch == '+') {
					ch = *p++;
				} else if (ch == '-') {
					s = -1;
					ch = *p++;
				}
				if (ch < '0' || ch > '9')
					expect("exponent digits");
				while (ch >= '0' && ch <= '9') {
					exp_val = exp_val * 10 + ch - '0';
					ch = *p++;
				}
				exp_val = exp_val * s;

				/* now we can generate the number */
				/* XXX: should patch directly float number */
				d = (double)bn[1] * 4294967296.0 + (double)bn[0];
				d = ldexp(d, exp_val - frac_bits);
				t = toup(ch);
				if (t == 'F') {
					ch = *p++;
					tok = TOK_CFLOAT;
					/* float : should handle overflow */
					tokc.f = (float)d;
				} else if (t == 'L') {
					ch = *p++;
					tok = TOK_CLDOUBLE;
					/* XXX: not large enough */
					tokc.ld = (long double)d;
				} else {
					tok = TOK_CDOUBLE;
					tokc.d = d;
				}
			} else {
				/* decimal floats */
				if (ch == '.') {
					if (q >= token_buf + STRING_MAX_SIZE)
						goto num_too_long;
					*q++ = ch;
					ch = *p++;
float_frac_parse:
					while (ch >= '0' && ch <= '9') {
						if (q >= token_buf + STRING_MAX_SIZE)
							goto num_too_long;
						*q++ = ch;
						ch = *p++;
					}
				}
				if (ch == 'e' || ch == 'E') {
					if (q >= token_buf + STRING_MAX_SIZE)
						goto num_too_long;
					*q++ = ch;
					ch = *p++;
					if (ch == '-' || ch == '+') {
						if (q >= token_buf + STRING_MAX_SIZE)
							goto num_too_long;
						*q++ = ch;
						ch = *p++;
					}
					if (ch < '0' || ch > '9')
						expect("exponent digits");
					while (ch >= '0' && ch <= '9') {
						if (q >= token_buf + STRING_MAX_SIZE)
							goto num_too_long;
						*q++ = ch;
						ch = *p++;
					}
				}
				*q = '\0';
				t = toup(ch);
				errno = 0;
				if (t == 'F') {
					ch = *p++;
					tok = TOK_CFLOAT;
					tokc.f = strtof(token_buf, NULL);
				} else if (t == 'L') {
					ch = *p++;
					tok = TOK_CLDOUBLE;
					tokc.ld = strtold(token_buf, NULL);
				} else {
					tok = TOK_CDOUBLE;
					tokc.d = strtod(token_buf, NULL);
				}
			}
	} else {
		unsigned long long n, n1;
		int lcount, ucount;

		/* integer number */
		*q = '\0';
		q = token_buf;
		if (b == 10 && *q == '0') {
			b = 8;
			q++;
		}
		n = 0;
		while(1) {
			t = *q++;
			/* no need for checks except for base 10 / 8 errors */
			if (t == '\0') {
				break;
			} else if (t >= 'a') {
				t = t - 'a' + 10;
			} else if (t >= 'A') {
				t = t - 'A' + 10;
			} else {
				t = t - '0';
				if (t >= b)
					error("invalid digit");
			}
			n1 = n;
			n = n * b + t;
			/* detect overflow */
			/* XXX: this test is not reliable */
			if (n < n1)
				error("integer constant overflow");
		}

		/* XXX: not exactly ANSI compliant */
		if ((n & 0xffffffff00000000LL) != 0) {
			if ((n >> 63) != 0)
				tok = TOK_CULLONG;
			else
				tok = TOK_CLLONG;
		} else if (n > 0x7fffffff) {
			tok = TOK_CUINT;
		} else {
			tok = TOK_CINT;
		}
		lcount = 0;
		ucount = 0;
		for(;;) {
			t = toup(ch);
			if (t == 'L') {
				if (lcount >= 2)
					error("three 'l's in integer constant");
				lcount++;
				if (lcount == 2) {
					if (tok == TOK_CINT)
						tok = TOK_CLLONG;
					else if (tok == TOK_CUINT)
						tok = TOK_CULLONG;
				}
				ch = *p++;
			} else if (t == 'U') {
				if (ucount >= 1)
					error("two 'u's in integer constant");
				ucount++;
				if (tok == TOK_CINT)
					tok = TOK_CUINT;
				else if (tok == TOK_CLLONG)
					tok = TOK_CULLONG;
				ch = *p++;
			} else {
				break;
			}
		}
		if (tok == TOK_CINT || tok == TOK_CUINT)
			tokc.ui = n;
		else
			tokc.ull = n;
	}
	if (ch)
		error("invalid number\n");
}


#define PARSE2(c1, tok1, c2, tok2)              \
	case c1:                                    \
	PEEKC(c, p);                            \
	if (c == c2) {                          \
	p++;                                \
	tok = tok2;                         \
	} else {                                \
	tok = tok1;                         \
	}                                       \
	break;

/* return next token without macro substitution */
static inline void next_nomacro1(void)
{
	int t, c, is_long;
	TokenSym *ts;
	uint8_t *p, *p1;
	unsigned int h;

	p = file->buf_ptr;
redo_no_start:
	c = *p;
	switch(c) 
	{
	case ' ':
	case '\t':
		tok = c;
		p++;
		goto keep_tok_flags;
	case '\f':
	case '\v':
	case '\r':
		p++;
		goto redo_no_start;
	case '\\':
		/* first look if it is in fact an end of buffer */
		if (p >= file->buf_end) 
		{
			file->buf_ptr = p;
			handle_eob();
			p = file->buf_ptr;
			if (p >= file->buf_end)
				goto parse_eof;
			else
				goto redo_no_start;
		} 
		else 
		{
			file->buf_ptr = p;
			ch = *p;
			handle_stray();
			p = file->buf_ptr;
			goto redo_no_start;
		}
parse_eof:
		{
			TCCState *s1 = tcc_state;
			if ((parse_flags & PARSE_FLAG_LINEFEED)
				&& !(tok_flags & TOK_FLAG_EOF)) 
			{
					tok_flags |= TOK_FLAG_EOF;
					tok = TOK_LINEFEED;
					goto keep_tok_flags;
			} 
			else if (s1->include_stack_ptr == s1->include_stack ||
				!(parse_flags & PARSE_FLAG_PREPROCESS)) 
			{
					/* no include left : end of file. */
					tok = TOK_EOF;
			} 
			else 
			{
				tok_flags &= ~TOK_FLAG_EOF;
				/* pop include file */

				/* test if previous '#endif' was after a #ifdef at
				start of file */
				if (tok_flags & TOK_FLAG_ENDIF) 
				{
#ifdef INC_DEBUG
					printf("#endif %s\n", get_tok_str(file->ifndef_macro_saved, NULL));
#endif
					add_cached_include(s1, file->inc_type, file->inc_filename,
						file->ifndef_macro_saved);
				}

				/* add end of include file debug info */
				if (tcc_state->do_debug) 
				{
					put_stabd(N_EINCL, 0, 0);
				}
				/* pop include stack */
				tcc_close(file);
				s1->include_stack_ptr--;
				file = *s1->include_stack_ptr;
				p = file->buf_ptr;
				goto redo_no_start;
			}
		}
		break;

	case '\n':
		file->line_num++;
		tok_flags |= TOK_FLAG_BOL;
		p++;
		if (0 == (parse_flags & PARSE_FLAG_LINEFEED))
			goto redo_no_start;
		tok = TOK_LINEFEED;
		goto keep_tok_flags;

	case '#':
		/* XXX: simplify */
		PEEKC(c, p);
		if ((tok_flags & TOK_FLAG_BOL) && 
			(parse_flags & PARSE_FLAG_PREPROCESS))
		{
				file->buf_ptr = p;
				preprocess(tok_flags & TOK_FLAG_BOF);
				p = file->buf_ptr;
				goto redo_no_start;
		} 
		else 
		{
			if (c == '#') 
			{
				p++;
				tok = TOK_TWOSHARPS;
			} 
			else 
			{
				if (parse_flags & PARSE_FLAG_ASM_COMMENTS)
				{
					p = parse_line_comment(p - 1);
					goto redo_no_start;
				} 
				else 
				{
					tok = '#';
				}
			}
		}
		break;

	case 'a': case 'b': case 'c': case 'd':
	case 'e': case 'f': case 'g': case 'h':
	case 'i': case 'j': case 'k': case 'l':
	case 'm': case 'n': case 'o': case 'p':
	case 'q': case 'r': case 's': case 't':
	case 'u': case 'v': case 'w': case 'x':
	case 'y': case 'z': 
	case 'A': case 'B': case 'C': case 'D':
	case 'E': case 'F': case 'G': case 'H':
	case 'I': case 'J': case 'K': 
	case 'M': case 'N': case 'O': case 'P':
	case 'Q': case 'R': case 'S': case 'T':
	case 'U': case 'V': case 'W': case 'X':
	case 'Y': case 'Z': 
	case '_':
parse_ident_fast:
		p1 = p;
		h = TOK_HASH_INIT;
		h = TOK_HASH_FUNC(h, c);
		p++;
		for(;;) 
		{
			c = *p;
			if (!isidnum_table[c-CH_EOF])
				break;
			h = TOK_HASH_FUNC(h, c);
			p++;
		}
		if (c != '\\') 
		{
			TokenSym **pts;
			int len;

			/* fast case : no stray found, so we have the full token
			and we have already hashed it */
			len = p - p1;
			h &= (TOK_HASH_SIZE - 1);
			pts = &hash_ident[h];
			for(;;) 
			{
				ts = *pts;
				if (!ts)
					break;
				if (ts->len == len && !memcmp(ts->str, p1, len))
					goto token_found;
				pts = &(ts->hash_next);
			}
			ts = tok_alloc_new(pts, p1, len);
token_found: ;
		} else {
			/* slower case */
			cstr_reset(&tokcstr);

			while (p1 < p) 
			{
				cstr_ccat(&tokcstr, *p1);
				p1++;
			}
			p--;
			PEEKC(c, p);
parse_ident_slow:
			while (isidnum_table[c-CH_EOF])
			{
				cstr_ccat(&tokcstr, c);
				PEEKC(c, p);
			}
			ts = tok_alloc(tokcstr.data, tokcstr.size);
		}
		tok = ts->tok;
		break;
	case 'L':
		t = p[1];
		if (t != '\\' && t != '\'' && t != '\"') 
		{
			/* fast case */
			goto parse_ident_fast;
		} else {
			PEEKC(c, p);
			if (c == '\'' || c == '\"') 
			{
				is_long = 1;
				goto str_const;
			} 
			else 
			{
				cstr_reset(&tokcstr);
				cstr_ccat(&tokcstr, 'L');
				goto parse_ident_slow;
			}
		}
		break;
	case '0': case '1': case '2': case '3':
	case '4': case '5': case '6': case '7':
	case '8': case '9':

		cstr_reset(&tokcstr);
		/* after the first digit, accept digits, alpha, '.' or sign if
		prefixed by 'eEpP' */
parse_num:
		for(;;) 
		{
			t = c;
			cstr_ccat(&tokcstr, c);
			PEEKC(c, p);
			if (!(isnum(c) || isid(c) || c == '.' ||
				((c == '+' || c == '-') && 
				(t == 'e' || t == 'E' || t == 'p' || t == 'P'))))
				break;
		}
		/* We add a trailing '\0' to ease parsing */
		cstr_ccat(&tokcstr, '\0');
		tokc.cstr = &tokcstr;
		tok = TOK_PPNUM;
		break;
	case '.':
		/* special dot handling because it can also start a number */
		PEEKC(c, p);
		if (isnum(c))
		{
			cstr_reset(&tokcstr);
			cstr_ccat(&tokcstr, '.');
			goto parse_num;
		} 
		else if (c == '.')
		{
			PEEKC(c, p);
			if (c != '.')
				expect("'.'");
			PEEKC(c, p);
			tok = TOK_DOTS;
		} 
		else 
		{
			tok = '.';
		}
		break;
	case '\'':
	case '\"':
		is_long = 0;
str_const:
		{
			CString str;
			int sep;

			sep = c;

			/* parse the string */
			cstr_new(&str);
			p = parse_pp_string(p, sep, &str);
			cstr_ccat(&str, '\0');

			/* eval the escape (should be done as TOK_PPNUM) */
			cstr_reset(&tokcstr);
			parse_escape_string(&tokcstr, str.data, is_long);
			cstr_free(&str);

			if (sep == '\'') 
			{
				int char_size;
				/* XXX: make it portable */
				if (!is_long)
					char_size = 1;
				else
					char_size = sizeof(nwchar_t);
				if (tokcstr.size <= char_size)
					error("empty character constant");
				if (tokcstr.size > 2 * char_size)
					warning("multi-character character constant");
				if (!is_long) 
				{
					tokc.i = *(int8_t *)tokcstr.data;
					tok = TOK_CCHAR;
				} 
				else 
				{
					tokc.i = *(nwchar_t *)tokcstr.data;
					tok = TOK_LCHAR;
				}
			} 
			else
			{
				tokc.cstr = &tokcstr;
				if (!is_long)
					tok = TOK_STR;
				else
					tok = TOK_LSTR;
			}
		}
		break;

	case '<':
		PEEKC(c, p);
		if (c == '=') 
		{
			p++;
			tok = TOK_LE;
		} 
		else if (c == '<') 
		{
			PEEKC(c, p);
			if (c == '=') 
			{
				p++;
				tok = TOK_A_SHL;
			} 
			else 
			{
				tok = TOK_SHL;
			}
		} 
		else 
		{
			tok = TOK_LT;
		}
		break;

	case '>':
		PEEKC(c, p);
		if (c == '=') 
		{
			p++;
			tok = TOK_GE;
		} 
		else if (c == '>') 
		{
			PEEKC(c, p);
			if (c == '=') 
			{
				p++;
				tok = TOK_A_SAR;
			} 
			else
			{
				tok = TOK_SAR;
			}
		} 
		else
		{
			tok = TOK_GT;
		}
		break;

	case '&':
		PEEKC(c, p);
		if (c == '&') 
		{
			p++;
			tok = TOK_LAND;
		} 
		else if (c == '=')
		{
			p++;
			tok = TOK_A_AND;
		} 
		else 
		{
			tok = '&';
		}
		break;

	case '|':
		PEEKC(c, p);
		if (c == '|')
		{
			p++;
			tok = TOK_LOR;
		} 
		else if (c == '=') 
		{
			p++;
			tok = TOK_A_OR;
		} else {
			tok = '|';
		}
		break;

	case '+':
		PEEKC(c, p);
		if (c == '+')
		{
			p++;
			tok = TOK_INC;
		} 
		else if (c == '=')
		{
			p++;
			tok = TOK_A_ADD;
		} else {
			tok = '+';
		}
		break;

	case '-':
		PEEKC(c, p);
		if (c == '-')
		{
			p++;
			tok = TOK_DEC;
		} 
		else if (c == '=')
		{
			p++;
			tok = TOK_A_SUB;
		}
		else if (c == '>')
		{
			p++;
			tok = TOK_ARROW;
		} else {
			tok = '-';
		}
		break;

		PARSE2('!', '!', '=', TOK_NE)
			PARSE2('=', '=', '=', TOK_EQ)
			PARSE2('*', '*', '=', TOK_A_MUL)
			PARSE2('%', '%', '=', TOK_A_MOD)
			PARSE2('^', '^', '=', TOK_A_XOR)

			/* comments or operator */
	case '/':
		PEEKC(c, p);
		if (c == '*') {
			p = parse_comment(p);
			goto redo_no_start;
		} else if (c == '/') {
			p = parse_line_comment(p);
			goto redo_no_start;
		} else if (c == '=') {
			p++;
			tok = TOK_A_DIV;
		} else {
			tok = '/';
		}
		break;

		/* simple tokens */
	case '(':
	case ')':
	case '[':
	case ']':
	case '{':
	case '}':
	case ',':
	case ';':
	case ':':
	case '?':
	case '~':
	case '$': /* only used in assembler */
	case '@': /* dito */
		tok = c;
		p++;
		break;
	default:
		error("unrecognized character \\x%02x", c);
		break;
	}
	tok_flags = 0;
keep_tok_flags:
	file->buf_ptr = p;
#if defined(PARSE_DEBUG)
	printf("token = %s\n", get_tok_str(tok, &tokc));
#endif
}

/* return next token without macro substitution. Can read input from
macro_ptr buffer */
static void next_nomacro_spc(void)
{
	if (macro_ptr) 
	{
redo:
		tok = *macro_ptr;
		if (tok) 
		{
			TOK_GET(tok, macro_ptr, tokc);
			if (tok == TOK_LINENUM) 
			{
				file->line_num = tokc.i;
				goto redo;
			}
		}
	} 
	else 
	{
		next_nomacro1();
	}
}

static void next_nomacro(void)
{
	do {
		next_nomacro_spc();
	} while (is_space(tok));
}

/* substitute args in macro_str and return allocated string */
static int *macro_arg_subst(Sym **nested_list, int *macro_str, Sym *args)
{
	int *st, last_tok, t, spc;
	Sym *s;
	CValue cval;
	TokenString str;
	CString cstr;

	tok_str_new(&str);
	last_tok = 0;
	while(1) {
		TOK_GET(t, macro_str, cval);
		if (!t)
			break;
		if (t == '#') {
			/* stringize */
			TOK_GET(t, macro_str, cval);
			if (!t)
				break;
			s = sym_find2(args, t);
			if (s) {
				cstr_new(&cstr);
				st = (int *)s->c;
				spc = 0;
				while (*st) {
					TOK_GET(t, st, cval);
					if (!check_space(t, &spc))
						cstr_cat(&cstr, get_tok_str(t, &cval));
				}
				cstr.size -= spc;
				cstr_ccat(&cstr, '\0');
#ifdef PP_DEBUG
				printf("stringize: %s\n", (char *)cstr.data);
#endif
				/* add string */
				cval.cstr = &cstr;
				tok_str_add2(&str, TOK_STR, &cval);
				cstr_free(&cstr);
			} else {
				tok_str_add2(&str, t, &cval);
			}
		} else if (t >= TOK_IDENT) {
			s = sym_find2(args, t);
			if (s) {
				st = (int *)s->c;
				/* if '##' is present before or after, no arg substitution */
				if (*macro_str == TOK_TWOSHARPS || last_tok == TOK_TWOSHARPS) {
					/* special case for var arg macros : ## eats the
					',' if empty VA_ARGS variable. */
					/* XXX: test of the ',' is not 100%
					reliable. should fix it to avoid security
					problems */
					if (gnu_ext && s->type.t &&
						last_tok == TOK_TWOSHARPS && 
						str.len >= 2 && str.str[str.len - 2] == ',') {
							if (*st == 0) {
								/* suppress ',' '##' */
								str.len -= 2;
							} else {
								/* suppress '##' and add variable */
								str.len--;
								goto add_var;
							}
					} else {
						int t1;
add_var:
						for(;;) {
							TOK_GET(t1, st, cval);
							if (!t1)
								break;
							tok_str_add2(&str, t1, &cval);
						}
					}
				} else {
					/* NOTE: the stream cannot be read when macro
					substituing an argument */
					macro_subst(&str, nested_list, st, NULL);
				}
			} else {
				tok_str_add(&str, t);
			}
		} else {
			tok_str_add2(&str, t, &cval);
		}
		last_tok = t;
	}
	tok_str_add(&str, 0);
	return str.str;
}

static char const ab_month_name[12][4] =
{
	"Jan", "Feb", "Mar", "Apr", "May", "Jun",
	"Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
};

/* do macro substitution of current token with macro 's' and add
result to (tok_str,tok_len). 'nested_list' is the list of all
macros we got inside to avoid recursing. Return non zero if no
substitution needs to be done */
static int macro_subst_tok(TokenString *tok_str,
						   Sym **nested_list, Sym *s, struct macro_level **can_read_stream)
{
	Sym *args, *sa, *sa1;
	int mstr_allocated, parlevel, *mstr, t, t1, *p, spc;
	TokenString str;
	char *cstrval;
	CValue cval;
	CString cstr;
	char buf[32];

	/* if symbol is a macro, prepare substitution */
	/* special macros */
	if (tok == TOK___LINE__) {
		snprintf(buf, sizeof(buf), "%d", file->line_num);
		cstrval = buf;
		t1 = TOK_PPNUM;
		goto add_cstr1;
	} else if (tok == TOK___FILE__) {
		cstrval = file->filename;
		goto add_cstr;
	} else if (tok == TOK___DATE__ || tok == TOK___TIME__) {
		time_t ti;
		struct tm *tm;

		time(&ti);
		tm = localtime(&ti);
		if (tok == TOK___DATE__) {
			snprintf(buf, sizeof(buf), "%s %2d %d", 
				ab_month_name[tm->tm_mon], tm->tm_mday, tm->tm_year + 1900);
		} else {
			snprintf(buf, sizeof(buf), "%02d:%02d:%02d", 
				tm->tm_hour, tm->tm_min, tm->tm_sec);
		}
		cstrval = buf;
add_cstr:
		t1 = TOK_STR;
add_cstr1:
		cstr_new(&cstr);
		cstr_cat(&cstr, cstrval);
		cstr_ccat(&cstr, '\0');
		cval.cstr = &cstr;
		tok_str_add2(tok_str, t1, &cval);
		cstr_free(&cstr);
	} else {
		mstr = (int *)s->c;
		mstr_allocated = 0;
		if (s->type.t == MACRO_FUNC) {
			/* NOTE: we do not use next_nomacro to avoid eating the
			next token. XXX: find better solution */
redo:
			if (macro_ptr) {
				p = macro_ptr;
				while (is_space(t = *p) || TOK_LINEFEED == t) 
					++p;
				if (t == 0 && can_read_stream) {
					/* end of macro stream: we must look at the token
					after in the file */
					struct macro_level *ml = *can_read_stream;
					macro_ptr = NULL;
					if (ml)
					{
						macro_ptr = ml->p;
						ml->p = NULL;
						*can_read_stream = ml -> prev;
					}
					goto redo;
				}
			} else {
				/* XXX: incorrect with comments */
				ch = file->buf_ptr[0];
				while (is_space(ch) || ch == '\n')
					cinp();
				t = ch;
			}
			if (t != '(') /* no macro subst */
				return -1;

			/* argument macro */
			next_nomacro();
			next_nomacro();
			args = NULL;
			sa = s->next;
			/* NOTE: empty args are allowed, except if no args */
			for(;;) {
				/* handle '()' case */
				if (!args && !sa && tok == ')')
					break;
				if (!sa)
					error("macro '%s' used with too many args",
					get_tok_str(s->v, 0));
				tok_str_new(&str);
				parlevel = spc = 0;
				/* NOTE: non zero sa->t indicates VA_ARGS */
				while ((parlevel > 0 || 
					(tok != ')' && 
					(tok != ',' || sa->type.t))) && 
					tok != -1) {
						if (tok == '(')
							parlevel++;
						else if (tok == ')')
							parlevel--;
						if (tok == TOK_LINEFEED)
							tok = ' ';
						if (!check_space(tok, &spc))
							tok_str_add2(&str, tok, &tokc);
						next_nomacro_spc();
				}
				str.len -= spc;
				tok_str_add(&str, 0);
				sym_push2(&args, sa->v & ~SYM_FIELD, sa->type.t, (long)str.str);
				sa = sa->next;
				if (tok == ')') {
					/* special case for gcc var args: add an empty
					var arg argument if it is omitted */
					if (sa && sa->type.t && gnu_ext)
						continue;
					else
						break;
				}
				if (tok != ',')
					expect(",");
				next_nomacro();
			}
			if (sa) {
				error("macro '%s' used with too few args",
					get_tok_str(s->v, 0));
			}

			/* now subst each arg */
			mstr = macro_arg_subst(nested_list, mstr, args);
			/* free memory */
			sa = args;
			while (sa) {
				sa1 = sa->prev;
				tok_str_free((int *)sa->c);
				sym_free(sa);
				sa = sa1;
			}
			mstr_allocated = 1;
		}
		sym_push2(nested_list, s->v, 0, 0);
		macro_subst(tok_str, nested_list, mstr, can_read_stream);
		/* pop nested defined symbol */
		sa1 = *nested_list;
		*nested_list = sa1->prev;
		sym_free(sa1);
		if (mstr_allocated)
			tok_str_free(mstr);
	}
	return 0;
}

/* handle the '##' operator. Return NULL if no '##' seen. Otherwise
return the resulting string (which must be freed). */
static inline int *macro_twosharps(const int *macro_str)
{
	TokenSym *ts;
	const int *ptr, *saved_macro_ptr;
	int t;
	const char *p1, *p2;
	CValue cval;
	TokenString macro_str1;
	CString cstr;

	/* we search the first '##' */
	for(ptr = macro_str;;) {
		TOK_GET(t, ptr, cval);
		if (t == TOK_TWOSHARPS)
			break;
		/* nothing more to do if end of string */
		if (t == 0)
			return NULL;
	}

	/* we saw '##', so we need more processing to handle it */
	cstr_new(&cstr);
	tok_str_new(&macro_str1);
	saved_macro_ptr = macro_ptr;
	/* XXX: get rid of the use of macro_ptr here */
	macro_ptr = (int *)macro_str;
	for(;;) {
		next_nomacro_spc();
		if (tok == 0)
			break;
		if (tok == TOK_TWOSHARPS)
			continue;
		while (*macro_ptr == TOK_TWOSHARPS) {
			t = *++macro_ptr;
			if (t && t != TOK_TWOSHARPS) {
				TOK_GET(t, macro_ptr, cval);
				/* We concatenate the two tokens if we have an
				identifier or a preprocessing number */
				cstr_reset(&cstr);
				p1 = get_tok_str(tok, &tokc);
				cstr_cat(&cstr, p1);
				p2 = get_tok_str(t, &cval);
				cstr_cat(&cstr, p2);
				cstr_ccat(&cstr, '\0');

				if ((tok >= TOK_IDENT || tok == TOK_PPNUM) && 
					(t >= TOK_IDENT || t == TOK_PPNUM)) {
						if (tok == TOK_PPNUM) {
							/* if number, then create a number token */
							/* NOTE: no need to allocate because
							tok_str_add2() does it */
							cstr_reset(&tokcstr);
							tokcstr = cstr;
							cstr_new(&cstr);
							tokc.cstr = &tokcstr;
						} else {
							/* if identifier, we must do a test to
							validate we have a correct identifier */
							if (t == TOK_PPNUM) {
								const char *p;
								int c;

								p = p2;
								for(;;) {
									c = *p;
									if (c == '\0')
										break;
									p++;
									if (!isnum(c) && !isid(c))
										goto error_pasting;
								}
							}
							ts = tok_alloc(cstr.data, strlen(cstr.data));
							tok = ts->tok; /* modify current token */
						}
				} else {
					const char *str = cstr.data;
					const unsigned char *q;

					/* we look for a valid token */
					/* XXX: do more extensive checks */
					if (!strcmp(str, ">>=")) {
						tok = TOK_A_SAR;
					} else if (!strcmp(str, "<<=")) {
						tok = TOK_A_SHL;
					} else if (strlen(str) == 2) {
						/* search in two bytes table */
						q = tok_two_chars;
						for(;;) {
							if (!*q)
								goto error_pasting;
							if (q[0] == str[0] && q[1] == str[1])
								break;
							q += 3;
						}
						tok = q[2];
					} else {
error_pasting:
						/* NOTE: because get_tok_str use a static buffer,
						we must save it */
						cstr_reset(&cstr);
						p1 = get_tok_str(tok, &tokc);
						cstr_cat(&cstr, p1);
						cstr_ccat(&cstr, '\0');
						p2 = get_tok_str(t, &cval);
						warning("pasting \"%s\" and \"%s\" does not give a valid preprocessing token", cstr.data, p2);
						/* cannot merge tokens: just add them separately */
						tok_str_add2(&macro_str1, tok, &tokc);
						/* XXX: free associated memory ? */
						tok = t;
						tokc = cval;
					}
				}
			}
		}
		tok_str_add2(&macro_str1, tok, &tokc);
	}
	macro_ptr = (int *)saved_macro_ptr;
	cstr_free(&cstr);
	tok_str_add(&macro_str1, 0);
	return macro_str1.str;
}


/* do macro substitution of macro_str and add result to
(tok_str,tok_len). 'nested_list' is the list of all macros we got
inside to avoid recursing. */
static void macro_subst(TokenString *tok_str, Sym **nested_list, 
						const int *macro_str, struct macro_level ** can_read_stream)
{
	Sym *s;
	int *macro_str1;
	const int *ptr;
	int t, ret, spc;
	CValue cval;
	struct macro_level ml;

	/* first scan for '##' operator handling */
	ptr = macro_str;
	macro_str1 = macro_twosharps(ptr);
	if (macro_str1) 
		ptr = macro_str1;
	spc = 0;
	while (1) {
		/* NOTE: ptr == NULL can only happen if tokens are read from
		file stream due to a macro function call */
		if (ptr == NULL)
			break;
		TOK_GET(t, ptr, cval);
		if (t == 0)
			break;
		s = define_find(t);
		if (s != NULL) {
			/* if nested substitution, do nothing */
			if (sym_find2(*nested_list, t))
				goto no_subst;
			ml.p = macro_ptr;
			if (can_read_stream)
				ml.prev = *can_read_stream, *can_read_stream = &ml;
			macro_ptr = (int *)ptr;
			tok = t;
			ret = macro_subst_tok(tok_str, nested_list, s, can_read_stream);
			ptr = (int *)macro_ptr;
			macro_ptr = ml.p;
			if (can_read_stream && *can_read_stream == &ml)
				*can_read_stream = ml.prev;
			if (ret != 0)
				goto no_subst;
		} else {
no_subst:
			if (!check_space(t, &spc)) 
				tok_str_add2(tok_str, t, &cval);
		}
	}
	if (macro_str1)
		tok_str_free(macro_str1);
}

/* return next token with macro substitution */
static void next(void)
{
	Sym *nested_list, *s;
	TokenString str;
	struct macro_level *ml;

redo:
	if (parse_flags & PARSE_FLAG_SPACES)
		next_nomacro_spc();
	else
		next_nomacro();
	if (!macro_ptr)  //have macro substition string
	{
		/* if not reading from macro substituted string, then try
		to substitute macros */
		if (tok >= TOK_IDENT &&
			(parse_flags & PARSE_FLAG_PREPROCESS)) 
		{
			s = define_find(tok);
			if (s) 
			{
				/* we have a macro: we try to substitute */
				tok_str_new(&str);
				nested_list = NULL;
				ml = NULL;
				if (macro_subst_tok(&str, &nested_list, s, &ml) == 0) //do macro substition
				{
					/* substitution done, NOTE: maybe empty */
					tok_str_add(&str, 0);
					macro_ptr = str.str;
					macro_ptr_allocated = str.str;
					goto redo;
				}
			}
		}
	} 
	else 
	{
		if (tok == 0) 
		{
			/* end of macro or end of unget buffer */
			if (unget_buffer_enabled) 
			{
				macro_ptr = unget_saved_macro_ptr;
				unget_buffer_enabled = 0;
			} 
			else 
			{
				/* end of macro string: free it */
				tok_str_free(macro_ptr_allocated);
				macro_ptr = NULL;
			}
			goto redo;
		}
	}

	/* convert preprocessor tokens into C tokens */
	if (tok == TOK_PPNUM &&
		(parse_flags & PARSE_FLAG_TOK_NUM)) 
	{
		parse_number((char *)tokc.cstr->data);
	}
}

/* push back current token and set current token to 'last_tok'. Only
identifier case handled for labels. */
static inline void unget_tok(int last_tok)
{
	int i, n;
	int *q;
	unget_saved_macro_ptr = macro_ptr;
	unget_buffer_enabled = 1;
	q = unget_saved_buffer;
	macro_ptr = q;
	*q++ = tok;
	n = tok_ext_size(tok) - 1;
	for(i=0;i<n;i++)
		*q++ = tokc.tab[i];
	*q = 0; /* end of token string */
	tok = last_tok;
}


/* better than nothing, but needs extension to handle '-E' option
correctly too */
static void preprocess_init(TCCState *s1)
{
	s1->include_stack_ptr = s1->include_stack;
	/* XXX: move that before to avoid having to initialize
	file->ifdef_stack_ptr ? */
	s1->ifdef_stack_ptr = s1->ifdef_stack;
	file->ifdef_stack_ptr = s1->ifdef_stack_ptr;

	/* XXX: not ANSI compliant: bound checking says error */
	vtop = vstack - 1;
	s1->pack_stack[0] = 0;
	s1->pack_stack_ptr = s1->pack_stack;
}

void preprocess_new()
{
	int i, c;
	const char *p, *r;
	TokenSym *ts;

	/* init isid table */
	for(i=CH_EOF;i<256;i++)
		isidnum_table[i-CH_EOF] = isid(i) || isnum(i);

	/* add all tokens */
	table_ident = NULL;
	memset(hash_ident, 0, TOK_HASH_SIZE * sizeof(TokenSym *));

	tok_ident = TOK_IDENT;
	p = tcc_keywords;
	while (*p) {
		r = p;
		for(;;) {
			c = *r++;
			if (c == '\0')
				break;
		}
		ts = tok_alloc(p, r - p - 1);
		p = r;
	}
}

/* Preprocess the current file */
static int tcc_preprocess(TCCState *s1)
{
	Sym *define_start;
	BufferedFile *file_ref;
	int token_seen, line_ref;

	preprocess_init(s1);
	define_start = define_stack;
	ch = file->buf_ptr[0];
	tok_flags = TOK_FLAG_BOL | TOK_FLAG_BOF;
	parse_flags = PARSE_FLAG_ASM_COMMENTS | PARSE_FLAG_PREPROCESS |
		PARSE_FLAG_LINEFEED | PARSE_FLAG_SPACES;
	token_seen = 0;
	line_ref = 0;
	file_ref = NULL;

	for (;;) {
		next();
		if (tok == TOK_EOF) {
			break;
		} else if (tok == TOK_LINEFEED) {
			if (!token_seen)
				continue;
			++line_ref;
			token_seen = 0;
		} else if (!token_seen) {
			int d = file->line_num - line_ref;
			if (file != file_ref || d < 0 || d >= 8)
				fprintf(s1->outfile, "# %d \"%s\"\n", file->line_num, file->filename);
			else
				while (d)
					fputs("\n", s1->outfile), --d;
			line_ref = (file_ref = file)->line_num;
			token_seen = 1;
		}
		fputs(get_tok_str(tok, &tokc), s1->outfile);
	}
	free_defines(define_start); 
	return 0;
}

